package com.example.demo;

import org.apache.log4j.Logger;
import org.thymeleaf.util.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

public class webMagicDemo implements PageProcessor {


    /**
     *
     * -- auto-generated definition
     * create table av_performer
     * (
     *     id       varchar(64)   not null comment '主键ID',
     *     name     varchar(500)  null comment '姓名',
     *     content  varchar(2000) null comment '简介',
     *     imgPath  varchar(1000) null comment '头像路径',
     *     birthday varchar(100)  null comment '出生日期',
     *     isWork   int           null comment '是否现役(0:是,1:否)',
     *     constraint av_performer_pk
     *         unique (id)
     * )
     *     comment '演员信息库';
     *
     * alter table av_performer
     *     add primary key (id);
     *
     *
     * -- auto-generated definition
     * create table av_works
     * (
     *     id         varchar(64)   not null comment '主键',
     *     name       varchar(2000) null comment '名称',
     *     performer  varchar(500)  null comment '演员',
     *     time       varchar(100)  null comment '上映时间',
     *     area       varchar(200)  null comment '上映地区',
     *     label      varchar(2000) null comment '文本标签类型',
     *     content    varchar(2000) null comment '作品简介',
     *     imgPath    varchar(2000) null comment '封面图片存储路径',
     *     localexist int(6)        null comment '服务器本地是否存在',
     *     localPath  varchar(2000) null comment '本地存在路径',
     *     constraint av_works_id_uindex
     *         unique (id)
     * )
     *     comment '番号作品库';
     *
     * alter table av_works
     *     add primary key (id);
     *
     */



    public static String dburl = "";
    public static String driver = "";
    public static String username = "";
    public static String password = "";

    private static List<String> urlList;
    private static List<String> url;
    private static List<String> url2;
    private static Logger log = Logger.getLogger(webMagicDemo.class);//log4j
    private static int number = 1;//主层初始页码数
    private static int endNumber = 47;//需要爬取的页码数 //最小为1

    private static int number2 = 0;//第二层序号
    private static int endNumber2 = 1;//第二层结束序号

    private static String MAIN_START_URL = "https://www.jsczc.com/nvyou/index.html";
    private static Spider spider = Spider.create(new webMagicDemo());
    private Site site = Site.me()
            .setDomain("www.baidu.com")
            .setSleepTime(1000)//爬取休眠时间
            .setCharset("utf-8")//编码集
            .setRetrySleepTime(3)//设置重试次数
            .setCycleRetryTimes(3)//设置循环重试次数
            .addHeader("Accept-Encoding","/")
            .setTimeOut(10000)//设置超时(网络不好尽可能调高,避免失败)
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");


    @Override
    public void process(Page page) {
        if(page.getUrl().regex("^https://www.jsczc.com/nvyou/index\\_\\d*$").match() || page.getUrl().regex("^https://www.jsczc.com/nvyou/index").match()){
            List<AvPerformer> paramList = new ArrayList<>();
            url = page.getHtml().xpath("*[@class=\"mt50\"]/ul/li/a").links().all();//下级URL
            List<String> performerName = page.getHtml().xpath("*[@class=\"mt50\"]/ul/li/a/p/text()").all();//演员姓名
            List<String> imgPath = page.getHtml().xpath("*[@class=\"mt50\"]/ul/li/a/img/@src").all();
            List<String> imgPath2 = new ArrayList<>();
            //由于网站的问题,拼接https:到图片的URL中,用于图片下载识别URL
            for (String url: imgPath) {
                imgPath2.add("https:" + url);
            }
            //组装第一层参数,包含头像,名称,下层url
            for (int i = 0;i<url.size();i++){
                AvPerformer avPerformer = new AvPerformer();
                avPerformer.setId(UUID.randomUUID().toString());
                avPerformer.setName(performerName.get(i));
                //创建主文件夹,下载头像
                String localImgPath = downloadPicture(imgPath2.get(i),avPerformer.getName(),"f:\\AV");
                avPerformer.setImgPath(localImgPath);
                //存入数据库
                StringBuilder sql  = new StringBuilder();
                sql.append("insert into av_performer(id, name, content, imgPath, birthday, isWork) values(")
                        .append("'").append(avPerformer.getId()).append("',")
                        .append("'").append(avPerformer.getName()).append("',")
                        .append("'").append(avPerformer.getContent()).append("',")
                        .append("'").append(avPerformer.getImgPath().replaceAll("\\\\","/")).append("',")
                        .append("'").append(avPerformer.getBirthday()).append("',")
                        .append("'").append(avPerformer.getIsWork()).append("'").append(")");
                try(Connection conn = DBConnection.getConnection(driver,dburl,username,password);) {
                    conn.prepareStatement(sql.toString()).execute();
                } catch (SQLException throwables) {
                    throwables.printStackTrace();
                }
            }
            //开始组装第一页的人员详情的第二层url组
            page.addTargetRequest(url.get(0));

        }else if(page.getUrl().regex("^https://www.jsczc.com/nvyou").match()){
            //这一层是演员的作品列表层
            String content = page.getHtml().xpath("*[@class=\"startext\"]/p/text()").toString();//演员简介
            url2 = page.getHtml().xpath("html/body/div[2]/ul[1]/li/a").links().all();//演员的所有作品明细页面
            if(url2.size()>0){
                page.addTargetRequest(url2.get(0));
            }else{
                if(url.size() > 0){
                    if(url.size() == 1){
                        page.addTargetRequest(url.get(0));
                        url.remove(0);
                        //爬取主列表下一页
                        if(number > endNumber+1){
                            return;
                        }else{
                            number++;
                            page.addTargetRequest(urlList.get(number-1));
                        }
                    }else{
                        url.remove(0);
                        page.addTargetRequest(url.get(0));
                    }
                }
            }
        }else if(page.getUrl().regex("https://www.jsczc.com/fanhao").match()){
            String imgUrl =page.getHtml().xpath("*[@class=\"content\"]/p/img/@src").toString();
            String name = page.getHtml().xpath("*[@class=\"avinfo\"]/p[1]/text()").toString();
            String performer = page.getHtml().xpath("*[@class=\"avinfo\"]/p[2]/text()").toString();
            String time = page.getHtml().xpath("*[@class=\"avinfo\"]/p[3]/text()").toString();
            String area = page.getHtml().xpath("*[@class=\"avinfo\"]/p[4]/text()").toString();
            String label= page.getHtml().xpath("*[@class=\"avinfo\"]/p[5]/text()").toString();
            String content = page.getHtml().xpath("*[@class=\"avinfo\"]/p[6]/text()").toString();
            String localImgPath = downloadPicture(imgUrl.replaceAll("http","https"),name,"f:\\AV\\" + performer);
            //进行数据库存储
            AvWorks avWorks = new AvWorks();
            avWorks.setId(UUID.randomUUID().toString());
            avWorks.setName(name);
            avWorks.setPerformer(performer);
            avWorks.setTime(time);
            avWorks.setArea(area);
            avWorks.setLabel(label);
            avWorks.setContent(content);
            avWorks.setImgPath(imgUrl);
            avWorks.setLocalPath(localImgPath);
            avWorks.setLocalexist(0);
            StringBuilder sql  = new StringBuilder();
            sql.append("insert into av_works(id,name,performer,time,area,label,content,imgPath,localexist,localPath) values(")
                    .append("'").append(avWorks.getId()).append("',")
                    .append("'").append(avWorks.getName()).append("',")
                    .append("'").append(avWorks.getPerformer()).append("',")
                    .append("'").append(avWorks.getTime()).append("',")
                    .append("'").append(avWorks.getArea()).append("',")
                    .append("'").append(avWorks.getLabel()).append("',")
                    .append("'").append(avWorks.getContent()).append("',")
                    .append("'").append(avWorks.getImgPath()).append("',")
                    .append("'").append(avWorks.getLocalexist()).append("',")
                    .append("'").append(avWorks.getLocalPath().replaceAll("\\\\","/")).append("'").append(")");
            try(Connection conn = DBConnection.getConnection(driver,dburl,username,password);) {
                conn.prepareStatement(sql.toString()).execute();
            } catch (SQLException throwables) {
                throwables.printStackTrace();
            }


            if(url2.size()>0){
                if(url2.size() == 1){
                    //当演员作品列表为1时,删除当前list的最后一个值,由于删除执行后,待爬取列表为null,所以需要补上下一个演员的列表详情url
                    page.addTargetRequest(url2.get(0));
                    url2.remove(0);

                    //补充下一个演员的列表
                    if(url.size() > 0){
                        if(url.size() == 1){
                            page.addTargetRequest(url.get(0));
                            url.remove(0);
                            //爬取主列表下一页
                            if(number > endNumber+1){
                                return;
                            }else{
                                number++;
                                page.addTargetRequest(urlList.get(number-1));
                            }
                        }else{
                            url.remove(0);
                            page.addTargetRequest(url.get(0));
                        }
                    }
                }else{
                    url2.remove(0);
                    page.addTargetRequest(url2.get(0));
                }
            }
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        urlList = doListUrl(endNumber);
        spider
                .thread(1)
                .addUrl(MAIN_START_URL)
                .run();
    }

    /**
     * 手工生成listUrl表
     */
    public static List<String> doListUrl(int num){
        List<String> list = new ArrayList<String>();
        for(int i = 1;i<=num+1;i++) {//页码从1开始
            list.add("https://www.jsczc.com/nvyou/index_" + (i+1) + ".html");
        }
        return list;
    }





    public String downloadPicture(String urlPath,String fileName,String filePath){
        if(StringUtils.isEmpty(urlPath)){//可能混杂一些空链接.需要处理
            return "";
        }
        try {
            URL url = new URL(urlPath);
            URLConnection con = url.openConnection();
            con.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
            InputStream inStream = con.getInputStream();
            ByteArrayOutputStream outStream = new ByteArrayOutputStream();
            byte[] buf = new byte[1024];
            int len = 0;
            while((len = inStream.read(buf)) != -1){
                outStream.write(buf,0,len);
            }
            inStream.close();
            outStream.close();
            File isexits = new File(filePath + "\\" + fileName);
            if(!isexits.exists())
            {
                isexits.mkdir();
            }
            /**
             *
             * 问题解决:outputstream流不能直接创建文件夹,所以在直接写不存在的文件夹的时候会出现问题,这个时候在前面进行一个文件夹是否存在的判断,
             * 若存在,直接存储不会出现问题,
             * 若不存在,需要先创建一个文件夹才可以
             */
            File file = new File(filePath + "\\" + fileName + "\\" +  fileName + ".jpg");
            FileOutputStream op = new FileOutputStream(file);
            op.write(outStream.toByteArray());
            op.flush();
            op.close();
            return filePath + "\\" + fileName + "\\" + fileName + ".jpg";
        }catch (Exception e){
            e.printStackTrace();
            log.error("下载图片出错,原因是" +e.getMessage());
            log.info("出错的链接是:" + fileName);
            return "";
        }
    }





    /**
     * 图片批量下载
     */
    public void downloadPictures(List<String> urlList,String fileName,String filePath) {
        for(int i = 0;i<urlList.size();i++){
            String link = urlList.get(i);//获取图片链接
            if(link == null || "".equals(link)){//可能混杂一些空链接.需要处理
                return;
            }
            try {
                URL url = new URL(link);
                URLConnection con = url.openConnection();
                con.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
                InputStream inStream = con.getInputStream();
                ByteArrayOutputStream outStream = new ByteArrayOutputStream();
                byte[] buf = new byte[1024];
                int len = 0;
                while((len = inStream.read(buf)) != -1){
                    outStream.write(buf,0,len);
                }
                inStream.close();
                outStream.close();
                File isexits = new File(filePath + "\\" + fileName);
                if(!isexits.exists())
                {
                    isexits.mkdir();
                }
                /**
                 *
                 * 问题解决:outputstream流不能直接创建文件夹,所以在直接写不存在的文件夹的时候会出现问题,这个时候在前面进行一个文件夹是否存在的判断,
                 * 若存在,直接存储不会出现问题,
                 * 若不存在,需要先创建一个文件夹才可以
                 */
                File file = new File(filePath + "\\" + fileName + "\\" +  fileName + ".jpg");
                FileOutputStream op = new FileOutputStream(file);
                op.write(outStream.toByteArray());
                op.close();
            }catch (Exception e){
                log.error("下载图片出错,原因是" +e.getMessage());
                log.info("出错的链接是:" + link);
            }
        }
    }
}
